#!/usr/bin/env python
import random
from mmap import mmap,ACCESS_READ
from xlrd import open_workbook
from utilities import Utilities 
from corpus_cleaner import CorpusCleaner
from Claim import Claim
import xml.dom.minidom
import numpy
from classifiers.SVMTreeExample import SVMTreeExample
import math
from feature_selection.BinormalSeparation import BinormalSeparation

"""
lee los datos, los limpia y los divide en datos de entrenamiento, validacion y pruebas
"""

class DatasetAcess:
	
	CATEGORY_COLUMN = 5
	DESCRIPTION_COlUMN = 10
	DATASET_PATH = ""
	TRAINING_SET_FILE = "../data/training_set.dat"
	VALIDATION_SET_FILE = "../data/validation_set.dat"
	TEST_SET_FILE = "../data/test_set.dat"
	TRAING_SET_PERCENT = 0.6
	VALIDATION_SET_PERCENT = 0.1
	TEST_SET_PERCENT = 0.3
	
	def __init__(self):
		self.training_set_path = DatasetAcess.TRAINING_SET_FILE
		self.validation_set_path = DatasetAcess.VALIDATION_SET_FILE
		self.test_set_path = DatasetAcess.TEST_SET_FILE

	def create_new_datasets(self):
		book = open_workbook('../data/Consolidado_de_quejas_2009-2012.xls')
		sheet = book.sheet_by_index(0)
		descriptions = sheet.col_values(DatasetAcess.DESCRIPTION_COlUMN)[2:]
		categories = sheet.col_values(DatasetAcess.CATEGORY_COLUMN)[2:]
		claims = {}

		cleaner = CorpusCleaner()
		for i in range(len(descriptions)):
			desc = str(descriptions[i].encode('utf-8'))
			c = str(categories[i].encode('utf-8'))
			desc = cleaner.clear_claim(desc)
			c = cleaner.clear_claim(c, False)
			c = c.replace(' ','_')
			print c
			try:
				claims[c].append( Claim(desc, c) )
			except KeyError:
				claims[c]=[ Claim(desc, c) ]

		self.print_frec_categories(claims)
		#group class with no enough claim
		mean = 18
		others = []
		#remove sistema bibliotecario
		"""for k in claims.keys():
			if (len(claims[k]))>30:
				del claims[k]"""
		keys = claims.keys()
		for category in keys:
			l = len(claims[category])
			if l<=30:
				#others.extend(claims[category])
				del claims[category]
				continue
			elif l>30 and l<=50:
				new_category= "minoriry"
				#del claims[category]
				#continue
			else:
				new_category= "mayority"
				#continue
			
			for e in claims[category]:
				e.set_meta_data(new_category)

		for o in others:
			o.set_category("others")
		if len(others)>0:
			claims['others']= others


		self.print_frec_categories(claims)
		
		self.create_features(claims)

		training_set = []
		validation_set = []
		test_set = []
		print 'numero de categories ', len(claims.keys())
		for category in claims.keys():
			docs = claims[category]
			random.shuffle(docs)

			i = 0
			j =int( len(docs)*DatasetAcess.TRAING_SET_PERCENT )
			training_set.extend(docs[i:j])

			i, j = j, j + int( len(docs)*DatasetAcess.VALIDATION_SET_PERCENT )
			validation_set.extend(docs[i:j])

			i, j = j, j + int( len(docs)*DatasetAcess.TEST_SET_PERCENT )
			test_set.extend(docs[i:j])

		random.shuffle(training_set)
		random.shuffle(validation_set)
		random.shuffle(test_set)
		
		v_training_set, v_validation_set, v_test_set = self.vectorizer_claims(training_set, validation_set, test_set)

		self.save_dataset(self.training_set_path, v_training_set)
		self.save_dataset(self.validation_set_path, v_validation_set)
		self.save_dataset(self.test_set_path, v_test_set)

	def save_dataset(self, file_name, dataset):
		raise NotImplementedError( "save_dataset method not implemented" )

	def get_training_set(self):
		return self.read_dataset(self.training_set_path)

	def get_validation_set(self):
		return self.read_dataset(self.validation_set_path)

	def get_test_set(self):
		return self.read_dataset(self.test_set_path)
	
	def get_dataset(self):
		dataset = []
		dataset.extend( self.get_training_set() )
		dataset.extend( self.get_validation_set() )
		dataset.extend( self.get_test_set() )
		return dataset

	def read_dataset(self, file_name):
		raise NotImplementedError( "read_dataset method not implemented" )
	
	def vectorizer_claims(self, training_set, validation_set, test_set):
		
		X=[]
		Y=[]
		meta_data = []
		l_training = len(training_set)
		l_validation = len(validation_set)
		l_test = len(test_set) 
		for example in training_set:
			X.append(example.vectorized_form())
			Y.append(example.get_category())
			meta_data.append(example.get_meta_data())
		for example in validation_set:
			X.append(example.vectorized_form())
			Y.append(example.get_category())
			meta_data.append(example.get_meta_data())
		for example in test_set:
			X.append(example.vectorized_form())
			Y.append(example.get_category())
			meta_data.append(example.get_meta_data())
		
		#X = self.apply_feature_selection(X, BinormalSeparation(X, meta_data))
		#X = self.normalize_data(X)
		
		m = len(X)
		examples =[]
		for i in range(m):
			e = SVMTreeExample(X[i], Y[i])
			e.set_upper_y(meta_data[i])
			examples.append(e)
		
		return examples[0:l_training], examples[l_training:l_training+l_validation], examples[l_training+l_validation: l_training+l_validation+l_test]
	
	
	def apply_feature_selection(self, X, selector):
		
		print 'applying feature selection...'
		indexes = selector.get_index_features()
		new_X = []
		for e in X:
			new_X.append( [f for index,f in enumerate(e) if index in indexes ] )
		return new_X
	
	def create_features(self, classes):
		print 'creating features...'
		c = classes.keys()
		print 'numero de clases', len(c)
		for current in c:
			rest_class = classes.keys()
			rest_class.remove(current)
			rest = []
			for list_claims in [classes[t] for t in rest_class ]:
				for claim in list_claims:
					rest.append(claim.vectorized_form())
			
			X =  [t.vectorized_form() for t in classes[current] ] +rest
			Y = [current]*len(classes[current]) + ['rest']*len(rest)
			#print 'length current ', len(classes[current])
			#print 'length rest ', len(rest)
			print 'current ', current
			selector = BinormalSeparation(X, Y)
			indexes = selector.get_index_features()
			output = open('../data/index_features/'+classes[current][0].get_category()+'.dat', "w")
			output.write(str(indexes))
			output.close()
			
	
	def print_frec_categories(self, claims):
		sum=0
		for c in claims:
			temp = len (claims[c])
			sum+=temp
			print c, " -> ", temp
		print 'suma de documentos: ', sum
			
